# coding=UTF-8
# 请求库
import requests
# 用于解决爬取的数据格式化
import io
import os
import sys
import datetime,time
import paramiko
# 用于处理json数据
import  json
import  ast
from io import StringIO

curPath = os.path.abspath(os.path.dirname(__file__))
# GetSeismicData，也就是项目的根路径
rootPath = curPath[:curPath.find("GetSeismicData\\") + len("GetSeismicData\\")]

# 获取数据存储文件路径
#dataPath = os.path.abspath(rootPath + 'data')+'\\'+datetime.datetime.now().strftime('%Y%m%d')+'_data.txt'
dataPath = os.path.abspath(rootPath + 'data')+'\\''data.txt'

# 获取记录时间文件路径
new_time_Path = os.path.abspath(rootPath + 'data\\new_time.log')

# 获取数据
def getdata():
    #sys.stdout = io.TextIOWrapper(sys.stdout.buffer, encoding='utf-8')
    # 调取最近24小时地震数据 api
    req = requests.get("http://www.ceic.ac.cn/ajax/speedsearch?num=2&&page=1")
    # 类型
    # print(type(req))
    # print(req.status_code)
    if req.status_code == 200:
        # 中文编码
        req.encoding = 'utf-8'
        # 将 JSON 对象转换为 Python 字典
        dict = ast.literal_eval(req.text)
        # 获取实际数据,返回 list
        date = dict["shuju"]
        print(date)
        # 判断 date 长度
        if len(date) >= 1:
            for i in reversed(date):
                    # print (i)
                    M = i["M"]  # 震级
                    O_TIME = i["O_TIME"]  # 发震时刻
                    EPI_LAT = i["EPI_LAT"]  # 纬度
                    EPI_LON = i["EPI_LON"]  # 经度
                    EPI_DEPTH = i["EPI_DEPTH"]  # 深度 km
                    LOCATION_C = i["LOCATION_C"]  # 参考位置
                    CREATE_TIME = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')
                    # 判断数据是否已经写入
                    time = file_read()
                    print("-------time 值: % s" % time)
                    #判断文件是否存在，不存在则创建
                    if bool(1-os.path.exists(dataPath)):
                        os.mknod(dataPath)
                    # 判断是否第一次写入
                    if time != "-1" and time is not None:
                        t1 = datetime.datetime.strptime(time, '%Y-%m-%d %H:%M:%S')
                        t2 = datetime.datetime.strptime(O_TIME, '%Y-%m-%d %H:%M:%S')
                        print("数据文件："+dataPath)
                        if t2 > t1:
                            # 覆盖写入数据
                            f_time = open(new_time_Path, mode='w', encoding='utf-8')
                            f_time.write(O_TIME)
                            #f_time.close()
                            # 测试为追加写入：mode='a'，正式为覆盖写入：mode='w'
                            f_date = open(dataPath, mode='a', encoding='utf-8')
                            f_date.write(
                                str(M) + ',' + str(O_TIME) + ',' + str(EPI_LAT) + ',' + str(EPI_LON) + ',' + str(
                                    EPI_DEPTH) + ',' + LOCATION_C + ',' + CREATE_TIME + '\n')

                    if time == '-1':
                        f_time = open(new_time_Path, mode='w', encoding='utf-8')
                        f_time.write(O_TIME)
                        #f_time.close()
                        f_date = open(dataPath, mode='w', encoding='utf-8')
                        f_date.write(str(M) + ',' + str(O_TIME) + ',' + str(EPI_LAT) + ',' + str(EPI_LON) + ',' + str(
                            EPI_DEPTH) + ',' + LOCATION_C + ',' + CREATE_TIME)

                        #f_date.close()

# 上传数据文件至服务器
# 服务器地址
hostname = "192.168.10.111"
# 账号
username = "root"
# 密码
pwd = "asd000000"
# 端口
port = 22
# local_file 本地文件路劲
# server_path 要上传的服务器路径
file_name = datetime.datetime.now().strftime('%Y%m%d')+'_data.txt'
server_path = os.path.join("/dataflie/",file_name)
shell_cmd = 'hive -e "use ods;LOAD DATA LOCAL INPATH \'' +server_path+'\' OVERWRITE  INTO TABLE ods_dl_seismic;"'
# 上传数据文件到服务器
def upload(local_file,server_path):
    try:
        ssh = paramiko.Transport((hostname,port))
        ssh.connect(username=username,password=pwd)
        sftp = paramiko.SFTPClient.from_transport(ssh)
        print("开始上传文件时间: %s"  % datetime.datetime.now())
        try:
            sftp.put(local_file,server_path)
        except Exception as e:
            sftp.mkdir(os.path.split(server_path)[0])
            sftp.put(local_file,server_path)
            print("从本地: %s 上传到: %s" % (local_file,server_path))
        print("文件上传成功: %s " % datetime.datetime.now())

        sftp.close()
        ssh.close()
    except Exception as e:
        print(repr(e))

# 文件读取
def file_read():
    try:
        with open(new_time_Path, 'r') as f:
            for line in f.readlines():
                #print("文件内容:" + line.strip())
                data = line.strip()
                return data

    except IOError:
        print("文件打开失败，%s文件不存在" % new_time_Path)

# 远程执行shell 加载外部数据到hive
class SshClient:
    "A wrapper of paramiko.SSHClient"
    TIMEOUT = 4

    def __init__(self, host, port, username, password, key=None, passphrase=None):
        self.username = username
        self.password = password
        self.client = paramiko.SSHClient()
        self.client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
        if key is not None:
            key = paramiko.RSAKey.from_private_key(StringIO(key), password=passphrase)
        self.client.connect(host, port, username=username, password=password, pkey=key, timeout=self.TIMEOUT)

    def close(self):
        if self.client is not None:
            self.client.close()
            self.client = None

    def execute(self, command, sudo=False):
        feed_password = False
        if sudo and self.username != "root":
            command = "sudo -S -p '' %s" % command
            feed_password = self.password is not None and len(self.password) > 0
        stdin, stdout, stderr = self.client.exec_command(command)
        if feed_password:
            stdin.write(self.password + "\n")
            stdin.flush()
        return {'out': stdout.readlines(),
                'err': stderr.readlines(),
                'retval': stdout.channel.recv_exit_status()}



if __name__ == "__main__":
    #getdata()
    #upload(dataPath, server_path)
    while True:
        print("-----------开始从地震网获取数据到本地-----------")
        getdata()
        print("-----------开始把本地数据上传至cdh服务器-----------")
        upload(dataPath, server_path)
        client = SshClient(host=hostname, port=22, username=username, password=pwd)
        try:
            print("-----------开始加载数据到hive-----------")
            ret = client.execute(shell_cmd, sudo=False)
            print("-----------获取hive执行结果-----------")
            print("  ".join(ret["out"]), "  E ".join(ret["err"]), ret["retval"])
        finally:
            client.close()
        #程序休眠 60S
        time.sleep(120)
















